# 有如下预测模型的的算法描述：
# 数据集：
# 类别不为空的用户描述表-训练用.excle
# 类别不为空的用户描述表-测试用.excle
# 类别为空的用户描述表-预测用.excle
# 三张表都包括用户id，用户文本描述,用户类别三列，只用 类别为空的用户描述表-预测用.excle 表中的类别为空
#
# 模式算法：
# 调用word2Vec模型使用，对三张表的第二列用户文本描述处理为特征向量，得到
# 类别不为空的用户特征向量表-训练用.excle
# 类别不为空的用户特征向量表-测试用.excle
# 类别为空的用户特征向量表-预测用.excle
#
# 调用深度神经网络（使用 torch，python3.8，numpy 进行编写）
# 使用类别不为空的用户特征向量表-训练用.excle训练模型
# 保存模型，命名为categoryModel
# 类别不为空的用户特征向量表-测试用.excle测试模型
# 输出模型的召回率，准确率指标
# 调用保存的模型，使用模型类别为空的用户特征向量表进行预测，输出类别不为空的用户特征向量表.excle
#
# 请根据上述预测模型的的算法描述编写相应的算法

##数据读取和预处理
import pandas as pd

# 读取Excel文件
train_df = pd.read_csv('train_user.csv')
test_df = pd.read_csv('test_user.csv')
predict_df = pd.read_csv('toref_user.csv')

# 提取文本描述列
train_descriptions = train_df['userDescription']
test_descriptions = test_df['userDescription']
predict_descriptions = predict_df['userDescription']

##使用Word2Vec模型生成特征向量
from gensim.models import Word2Vec
import numpy as np

# 对描述进行分词（假设每个描述已经是一个分词后的列表）
train_descriptions = train_descriptions.apply(lambda x: x.split())
test_descriptions = test_descriptions.apply(lambda x: x.split())
predict_descriptions = predict_descriptions.apply(lambda x: x.split())

# 合并所有描述以便训练Word2Vec模型
all_descriptions = pd.concat([train_descriptions, test_descriptions, predict_descriptions])

# 训练Word2Vec模型
model = Word2Vec(all_descriptions, vector_size=100, window=5, min_count=1, workers=4)

# 将描述转换为特征向量
def get_feature_vector(description, model):
    words = description
    feature_vector = np.mean([model.wv[word] for word in words if word in model.wv], axis=0)
    return feature_vector

train_vectors = train_descriptions.apply(lambda x: get_feature_vector(x, model))
test_vectors = test_descriptions.apply(lambda x: get_feature_vector(x, model))
predict_vectors = predict_descriptions.apply(lambda x: get_feature_vector(x, model))

# # 保存特征向量到Excel文件
# train_vectors_df = pd.DataFrame(train_vectors.tolist())
# test_vectors_df = pd.DataFrame(test_vectors.tolist())
# predict_vectors_df = pd.DataFrame(predict_vectors.tolist())
#
# train_vectors_df.to_excel('类别不为空的用户特征向量表-训练用.excel', index=False)
# test_vectors_df.to_excel('类别不为空的用户特征向量表-测试用.excel', index=False)
# predict_vectors_df.to_excel('类别为空的用户特征向量表-预测用.excel', index=False)

##构建和训练深度神经网络模型
import torch
import torch.nn as nn
import torch.optim as optim


# 定义深度神经网络模型
class CategoryModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(CategoryModel, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        out = self.fc1(x)
        out = self.relu(out)
        out = self.fc2(out)
        return out


# 参数设置
input_dim = 100  # Word2Vec向量的维度
hidden_dim = 64
output_dim = train_df['category'].nunique()  # 类别数量
num_epochs = 20
learning_rate = 0.001

# 数据准备
train_X = np.stack(train_vectors.values)
train_y = train_df['category'].values
test_X = np.stack(test_vectors.values)
test_y = test_df['category'].values

# 转换为PyTorch张量
train_X = torch.tensor(train_X, dtype=torch.float32)
train_y = torch.tensor(train_y, dtype=torch.long)
test_X = torch.tensor(test_X, dtype=torch.float32)
test_y = torch.tensor(test_y, dtype=torch.long)

# 模型实例化
model = CategoryModel(input_dim, hidden_dim, output_dim)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

# 模型训练
for epoch in range(num_epochs):
    model.train()
    outputs = model(train_X)
    loss = criterion(outputs, train_y)
#https://chatgpt.com/c/ae62d168-eee5-415f-91a2-bf684eae6521
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    print(f'Epoch [{epoch + 1}/{num_epochs}], Loss: {loss.item():.4f}')

# 保存模型
torch.save(model.state_dict(), 'categoryModel.pth')

##模型测试和评估
from sklearn.metrics import accuracy_score, recall_score

# 模型测试
model.eval()
with torch.no_grad():
    test_outputs = model(test_X)
    _, predicted = torch.max(test_outputs, 1)

# 计算指标
accuracy = accuracy_score(test_y, predicted)
recall = recall_score(test_y, predicted, average='weighted')

print(f'Accuracy: {accuracy:.4f}')
print(f'Recall: {recall:.4f}')

##使用保存的模型进行预测

# 载入模型
model = CategoryModel(input_dim, hidden_dim, output_dim)
model.load_state_dict(torch.load('categoryModel.pth'))
model.eval()

# 进行预测
predict_X = np.stack(predict_vectors.values)
predict_X = torch.tensor(predict_X, dtype=torch.float32)

with torch.no_grad():
    predict_outputs = model(predict_X)
    _, predicted_categories = torch.max(predict_outputs, 1)

# 输出预测结果
predict_df['category'] = predicted_categories.numpy()
predict_df.to_csv('toref_user.csv', index=False)

